#!/usr/bin/env python
# encoding=utf-8

import sys

from pymongo import MongoClient
from scpy.logger import get_logger
from xtls.mqwrapper import producer

from lagou_job_crawler import LagouCrawler

reload(sys)
sys.setdefaultencoding('utf-8')

__author__ = 'xlzd'
logger = get_logger(__file__)
MONGO = MongoClient('10.132.23.104', 27017)


def main():
    """
    直接逐条抓取（慎用！）
    """
    for index, item in enumerate(MONGO['crawler_company_all']['lagouCompany'].find().batch_size(1)):
        cid = int(item['srcUrl'][28:-5])
        logger.info('now company %s %s' % (item['companyName'], cid))
        LagouCrawler(cid, item['companyName']).run()


@producer('10.117.29.191', 'lagou-company', logger=logger)
def insert_task():
    """
    向MQ中插入任务
    """
    for index, item in enumerate(MONGO['crawler_company_all']['lagouCompany'].find().batch_size(1)):
        cid = int(item['srcUrl'][28:-5])
        logger.info('now company %s %s' % (item['companyName'], cid))
        yield {'cid': cid, 'cname': item['companyName']}


if __name__ == '__main__':
    insert_task()
